import pickle
import helpsk as hlp
import matplotlib.pyplot as plt
import pandas as pd
import numpy as np
from sklearn.datasets import fetch_openml
import seaborn as sns

from sklearn.pipeline import Pipeline
from sklearn.preprocessing import StandardScaler, MinMaxScaler, OneHotEncoder  # , LabelEncoder
from sklearn.impute import SimpleImputer
from sklearn.model_selection import train_test_split
# from sklearn.base import BaseEstimator, TransformerMixin

%matplotlib inline

Load Data

# with open('../X_train.pkl', 'rb') as handle:
#     X_train = pickle.load(handle)
#     print(X_train.shape)

# with open('../y_train.pkl', 'rb') as handle:
#     y_train = pickle.load(handle)
#     print(len(y_train))

with open('X_test.pkl', 'rb') as handle:
    X_test = pickle.load(handle)
    print(X_test.shape)

with open('y_test.pkl', 'rb') as handle:
    y_test = pickle.load(handle)
    print(len(y_test))

(200, 20)
200

X_test.head()

	checking_status	duration	credit_history	purpose	credit_amount	savings_status	employment	installment_commitment	personal_status	other_parties	residence_since	property_magnitude	age	other_payment_plans	housing	existing_credits	job	num_dependents	own_telephone	foreign_worker
521	<0	18.0	existing paid	radio/tv	3190.0	<100	1<=X<4	2.0	female div/dep/mar	none	2.0	real estate	24.0	none	own	1.0	skilled	1.0	none	yes
737	<0	18.0	existing paid	new car	4380.0	100<=X<500	1<=X<4	3.0	male single	none	4.0	car	35.0	none	own	1.0	unskilled resident	2.0	yes	yes
740	<0	24.0	all paid	new car	2325.0	100<=X<500	4<=X<7	2.0	male single	none	3.0	car	32.0	bank	own	1.0	skilled	1.0	none	yes
660	>=200	12.0	existing paid	radio/tv	1297.0	<100	1<=X<4	3.0	male mar/wid	none	4.0	real estate	23.0	none	rent	1.0	skilled	1.0	none	yes
411	no checking	33.0	critical/other existing credit	used car	7253.0	<100	4<=X<7	3.0	male single	none	2.0	car	35.0	none	own	2.0	high qualif/self emp/mgmt	1.0	yes	yes

y_test[0:10]

array([1, 0, 0, 0, 0, 0, 0, 0, 0, 0])

Load Test Set Predictions from Models

with open('Logistic Regression/test_set_predictions.pkl', 'rb') as handle:
    logistic_predictions = pickle.load(handle)

logistic_predictions[0:10]

array([0.40677117, 0.50960321, 0.61519753, 0.32037735, 0.07595768,
       0.36274643, 0.0779174 , 0.4667194 , 0.0829285 , 0.11700064])

with open('XGBoost/test_set_predictions.pkl', 'rb') as handle:
    xgb_predictions = pickle.load(handle)

xgb_predictions[0:10]

array([0.33118248, 0.59089786, 0.6703654 , 0.39817235, 0.07021371,
       0.41777927, 0.06150061, 0.45194232, 0.10988432, 0.21019126],
      dtype=float32)

with open('Stacking/test_set_predictions.pkl', 'rb') as handle:
    stacking_predictions = pickle.load(handle)

stacking_predictions[0:10]

array([0.49853447, 0.50046754, 0.5013185 , 0.49942148, 0.4958812 ,
       0.4995596 , 0.4958812 , 0.49975598, 0.49589697, 0.49672192],
      dtype=float32)

Model Comparison

from helpsk.sklearn_eval import TwoClassModelComparison

predictions = {
    'Logistic Regression': logistic_predictions,
    'XGBoost': xgb_predictions,
    'Stacking': stacking_predictions,
}

comparison = TwoClassModelComparison(
    actual_values=y_test,
    predicted_scores=predictions, 
)

comparison.all_metrics_df(
    dummy_classifier_strategy=['prior', 'constant'],
    return_style=True,
    round_by=2
)

	AUC	F1 Score	True Positive Rate	True Negative Rate	False Positive Rate	False Negative Rate	Positive Predictive Value	Negative Predictive Value
Logistic Regression	0.80	0.59	0.51	0.91	0.09	0.49	0.70	0.82
XGBoost	0.81	0.61	0.51	0.94	0.06	0.49	0.77	0.82
Stacking	0.80	0.57	0.47	0.91	0.09	0.53	0.70	0.81
Dummy (prior)	0.50	0.00	0.00	1.00	0.00	1.00	0.00	0.70
Dummy (constant)	0.50	0.46	1.00	0.00	1.00	0.00	0.30	0.00

comparison.plot_metrics_comparison()

comparison.plot_roc_curves()